{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## API tutorial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expression building\n",
    "\n",
    "(note: may have old API in some cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from dynet import *\n",
    "\n",
    "## ==== Create a new computation graph\n",
    "# (it is a singleton, we have one at each stage.\n",
    "# renew_cg() clears the current one and starts anew)\n",
    "renew_cg()\n",
    "\n",
    "## ==== Creating Expressions from user input / constants.\n",
    "x = scalarInput(value)\n",
    "\n",
    "v = vecInput(dimension)\n",
    "v.set([1,2,3])\n",
    "\n",
    "z = matInput(dim1, dim2)\n",
    "\n",
    "# for example:\n",
    "z1 = matInput(2, 2)\n",
    "z1.set([1,2,3,4]) # Column major\n",
    "\n",
    "# Or directly from a numpy array\n",
    "z1 = inputTensor([[1,2],[3,4]]) # Row major\n",
    "\n",
    "## ==== We can take the value of an expression. \n",
    "# For complex expressions, this will run forward propagation.\n",
    "print z.value()    \n",
    "print z.npvalue()      # as numpy array\n",
    "print v.vec_value()    # as vector, if vector\n",
    "print x.scalar_value() # as scalar, if scalar\n",
    "print x.value()        # choose the correct one\n",
    "\n",
    "## ==== Parameters\n",
    "# Parameters are things we tune during training.\n",
    "# Usually a matrix or a vector.\n",
    "\n",
    "# First we create a parameter collection and add the parameters to it.\n",
    "m = ParameterCollection() \n",
    "pW = m.add_parameters((8,8)) # an 8x8 matrix\n",
    "pb = m.add_parameters(8)\n",
    "\n",
    "# then we create an Expression out of the parameter collection's parameters\n",
    "W = parameter(pW)\n",
    "b = parameter(pb)\n",
    "\n",
    "## ===== Lookup parameters\n",
    "# Similar to parameters, but are representing a \"lookup table\"\n",
    "# that maps numbers to vectors.\n",
    "# These are used for embedding matrices.\n",
    "# for example, this will have VOCAB_SIZE rows, each of DIM dimensions.\n",
    "lp  = m.add_lookup_parameters((VOCAB_SIZE, DIM))\n",
    "\n",
    "# lookup parameters can be initialized from an existing array, i.e:\n",
    "# m[\"lookup\"].init_from_array(wv)\n",
    "\n",
    "e5  = lookup(lp, 5)   # create an Expression from row 5.\n",
    "e5  = lp[5]           # same\n",
    "e5c = lookup(lp, 5, update=False)  # as before, but don't update when optimizing.\n",
    "\n",
    "e5  = lookup_batch(lp, [4, 5])   # create a batched Expression from rows 4 and 5.\n",
    "e5  = lp.batch([4, 5])           # same\n",
    "\n",
    "e5.set(9)   # now the e5 expression contains row 9\n",
    "e5c.set(9)  # ditto\n",
    "\n",
    "\n",
    "## ===== Combine expression into complex expressions.\n",
    "\n",
    "# Math \n",
    "e = e1 + e2   \n",
    "e = e1 * e2   # for vectors/matrices: matrix multiplication (like e1.dot(e2) in numpy)\n",
    "e = e1 - e2    \n",
    "e = -e1 \n",
    "\n",
    "e = dot_product(e1, e2)\n",
    "e = cmult(e1, e2)           # component-wise multiply  (like e1*e2 in numpy)\n",
    "e = cdiv(e1, e2)            # component-wise divide\n",
    "e = colwise_add(e1, e2)     # column-wise addition\n",
    "\n",
    "# Matrix Shapes\n",
    "e = reshape(e1, new_dimension)\n",
    "e = transpose(e1)\n",
    "\n",
    "# Per-element unary functions.\n",
    "e = tanh(e1)      \n",
    "e = exp(e1)\n",
    "e = log(e1)\n",
    "e = logistic(e1)   # Sigmoid(x)\n",
    "e = rectify(e1)    # Relu (= max(x,0))\n",
    "e = softsign(e1)    # x/(1+|x|)\n",
    "\n",
    "# softmaxes\n",
    "e = softmax(e1)\n",
    "e = log_softmax(e1, restrict=[]) # restrict is a set of indices. \n",
    "                                 # if not empty, only entries in restrict are part \n",
    "                                 # of softmax computation, others get 0.\n",
    "\n",
    "\n",
    "e = sum_cols(e1)\n",
    "\n",
    "\n",
    "# Picking values from vector expressions\n",
    "e = pick(e1, k)              # k is unsigned integer, e1 is vector. return e1[k]\n",
    "e = e1[k]                    # same\n",
    "\n",
    "e = pickrange(e1, k, v)      # like python's e1[k:v] for lists. e1 is an Expression, k,v integers.\n",
    "e = e1[k:v]                  # same\n",
    "\n",
    "e = pickneglogsoftmax(e1, k) # k is unsigned integer. equiv to: (pick(-log(softmax(e1)), k))\n",
    "                             \n",
    "\n",
    "# Neural net stuff\n",
    "noise(e1, stddev) # add a noise to each element from a gausian with standard-dev = stddev\n",
    "dropout(e1, p)    # apply dropout with probability p \n",
    "\n",
    "# functions over lists of expressions\n",
    "e = esum([e1, e2, ...])            # sum\n",
    "e = average([e1, e2, ...])         # average\n",
    "e = concatenate_cols([e1, e2, ...])  # e1, e2,.. are column vectors. return a matrix. (sim to np.hstack([e1,e2,...])\n",
    "e = concatenate([e1, e2, ...])     # concatenate\n",
    "\n",
    "e = affine_transform([e0,e1,e2, ...])  # e = e0 + ((e1*e2) + (e3*e4) ...) \n",
    "\n",
    "## Loss functions\n",
    "e = squared_distance(e1, e2)\n",
    "e = l1_distance(e1, e2)\n",
    "e = huber_distance(e1, e2, c=1.345)\n",
    "\n",
    "# e1 must be a scalar that is a value between 0 and 1\n",
    "# e2 (ty) must be a scalar that is a value between 0 and 1\n",
    "# e = ty * log(e1) + (1 - ty) * log(1 - e1)\n",
    "e = binary_log_loss(e1, e2)\n",
    "\n",
    "# e1 is row vector or scalar\n",
    "# e2 is row vector or scalar\n",
    "# m is number\n",
    "# e = max(0, m - (e1 - e2))\n",
    "e = pairwise_rank_loss(e1, e2, m=1.0) \n",
    "\n",
    "# Convolutions\n",
    "# e1 \\in R^{d x s} (input)\n",
    "# e2 \\in R^{d x m} (filter)\n",
    "e = conv1d_narrow(e1, e2) # e = e1 *conv e2\n",
    "e = conv1d_wide(e1, e2)   # e = e1 *conv e2\n",
    "e = filter1d_narrow(e1, e2) # e = e1 *filter e2\n",
    "\n",
    "e = kmax_pooling(e1, k) #  kmax-pooling operation (Kalchbrenner et al 2014)\n",
    "e = kmh_ngram(e1, k) # \n",
    "e = fold_rows(e1, nrows=2) #\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "added\n",
      "[1, 2, 3] 1\n",
      "2.71492385864\n",
      "[3, 2, 4] 2\n",
      "2.48228144646\n",
      "[1, 2, 3] 1\n",
      "2.00279903412\n",
      "[3, 2, 4] 2\n",
      "1.82602763176\n",
      "[1, 2, 3] 1\n",
      "1.44809651375\n",
      "[3, 2, 4] 2\n",
      "1.34181213379\n",
      "[1, 2, 3] 1\n",
      "1.03570735455\n",
      "[3, 2, 4] 2\n",
      "0.988352060318\n",
      "[1, 2, 3] 1\n",
      "0.744616270065\n",
      "[3, 2, 4] 2\n",
      "0.732948303223\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "from dynet import *\n",
    "\n",
    "# create parameter collection\n",
    "m = ParameterCollection()\n",
    "\n",
    "# add parameters to parameter collection\n",
    "pW = m.add_parameters((10,30))\n",
    "pB = m.add_parameters(10)\n",
    "lookup = m.add_lookup_parameters((500, 10))\n",
    "print \"added\"\n",
    "\n",
    "# create trainer \n",
    "trainer = SimpleSGDTrainer(m)\n",
    "\n",
    "# Regularization is set via the --dynet-l2 commandline flag.\n",
    "# Learning rate parameters can be passed to the trainer:\n",
    "# alpha = 0.1  # learning rate\n",
    "# trainer = SimpleSGDTrainer(m, e0=alpha)\n",
    "\n",
    "# function for graph creation\n",
    "def create_network_return_loss(inputs, expected_output):\n",
    "    \"\"\"\n",
    "    inputs is a list of numbers\n",
    "    \"\"\"\n",
    "    renew_cg()\n",
    "    W = parameter(pW) # from parameters to expressions\n",
    "    b = parameter(pB)\n",
    "    emb_vectors = [lookup[i] for i in inputs]\n",
    "    net_input = concatenate(emb_vectors)\n",
    "    net_output = softmax( (W*net_input) + b)\n",
    "    loss = -log(pick(net_output, expected_output))\n",
    "    return loss\n",
    "\n",
    "# function for prediction\n",
    "def create_network_return_best(inputs):\n",
    "    \"\"\"\n",
    "    inputs is a list of numbers\n",
    "    \"\"\"\n",
    "    renew_cg()\n",
    "    W = parameter(pW)\n",
    "    b = parameter(pB)\n",
    "    emb_vectors = [lookup[i] for i in inputs]\n",
    "    net_input = concatenate(emb_vectors)\n",
    "    net_output = softmax( (W*net_input) + b)\n",
    "    return np.argmax(net_output.npvalue())\n",
    "\n",
    "\n",
    "# train network\n",
    "for epoch in xrange(5):\n",
    "    for inp,lbl in ( ([1,2,3],1), ([3,2,4],2) ):\n",
    "        print inp, lbl\n",
    "        loss = create_network_return_loss(inp, lbl)\n",
    "        print loss.value() # need to run loss.value() for the forward prop\n",
    "        loss.backward()\n",
    "        trainer.update()\n",
    "\n",
    "print create_network_return_best([1,2,3])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Recipe (using classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3] 1\n",
      "2.5900914669\n",
      "[3, 2, 4] 2\n",
      "2.00347089767\n",
      "[1, 2, 3] 1\n",
      "1.98409461975\n",
      "[3, 2, 4] 2\n",
      "1.50869822502\n",
      "[1, 2, 3] 1\n",
      "1.50195622444\n",
      "[3, 2, 4] 2\n",
      "1.12316584587\n",
      "[1, 2, 3] 1\n",
      "1.12293696404\n",
      "[3, 2, 4] 2\n",
      "0.831095397472\n",
      "[1, 2, 3] 1\n",
      "0.833912611008\n",
      "[3, 2, 4] 2\n",
      "0.61754822731\n",
      "\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "from dynet import *\n",
    "# create parameter collection\n",
    "m = ParameterCollection()\n",
    "\n",
    "# create a class encapsulating the network\n",
    "class OurNetwork(object):\n",
    "    # The init method adds parameters to the parameter collection.\n",
    "    def __init__(self, pc):\n",
    "        self.pW = pc.add_parameters((10,30))\n",
    "        self.pB = pc.add_parameters(10)\n",
    "        self.lookup = pc.add_lookup_parameters((500,10))\n",
    "    \n",
    "    # the __call__ method applies the network to an input\n",
    "    def __call__(self, inputs):\n",
    "        W = parameter(self.pW)\n",
    "        b = parameter(self.pB)\n",
    "        lookup = self.lookup\n",
    "        emb_vectors = [lookup[i] for i in inputs]\n",
    "        net_input = concatenate(emb_vectors)\n",
    "        net_output = softmax( (W*net_input) + b)\n",
    "        return net_output\n",
    "    \n",
    "    def create_network_return_loss(self, inputs, expected_output):\n",
    "        renew_cg()\n",
    "        out = self(inputs)\n",
    "        loss = -log(pick(out, expected_output))\n",
    "        return loss\n",
    "       \n",
    "    def create_network_return_best(self, inputs):\n",
    "        renew_cg()\n",
    "        out = self(inputs)\n",
    "        return np.argmax(out.npvalue())\n",
    "        \n",
    "        \n",
    "# create network\n",
    "network = OurNetwork(m)\n",
    "\n",
    "# create trainer \n",
    "trainer = SimpleSGDTrainer(m)\n",
    "   \n",
    "# train network\n",
    "for epoch in xrange(5):\n",
    "    for inp,lbl in ( ([1,2,3],1), ([3,2,4],2) ):\n",
    "        print inp, lbl\n",
    "        loss = network.create_network_return_loss(inp, lbl)\n",
    "        print loss.value() # need to run loss.value() for the forward prop\n",
    "        loss.backward()\n",
    "        trainer.update()\n",
    "\n",
    "print\n",
    "print network.create_network_return_best([1,2,3])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### or, alternatively, have the training outside of the network class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3] 1\n",
      "3.63615298271\n",
      "[3, 2, 4] 2\n",
      "3.29473733902\n",
      "[1, 2, 3] 1\n",
      "2.81605744362\n",
      "[3, 2, 4] 2\n",
      "2.46070289612\n",
      "[1, 2, 3] 1\n",
      "2.13946056366\n",
      "[3, 2, 4] 2\n",
      "1.77259361744\n",
      "[1, 2, 3] 1\n",
      "1.57904195786\n",
      "[3, 2, 4] 2\n",
      "1.2269589901\n",
      "[1, 2, 3] 1\n",
      "1.13014268875\n",
      "[3, 2, 4] 2\n",
      "0.830479979515\n",
      "\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "# create network\n",
    "network = OurNetwork(m)\n",
    "\n",
    "# create trainer \n",
    "trainer = SimpleSGDTrainer(m)\n",
    "   \n",
    "# train network\n",
    "for epoch in xrange(5):\n",
    "    for inp,lbl in ( ([1,2,3],1), ([3,2,4],2) ):\n",
    "        print inp, lbl\n",
    "        renew_cg()\n",
    "        out = network(inp)\n",
    "        loss = -log(pick(out, lbl))\n",
    "        print loss.value() # need to run loss.value() for the forward prop\n",
    "        loss.backward()\n",
    "        trainer.update()\n",
    "\n",
    "print\n",
    "print np.argmax(network([1,2,3]).npvalue())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
